#!/usr/bin/env python
import re
from scrapy import Request, Spider
from bots.spiderbot.spiderbot.items import BatteryItem

# from faker import Faker
from selenium import webdriver
# from selenium.common.exceptions import TimeoutException
# from selenium.webdriver.common.by import By
# from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
# from scrapy.http import HtmlResponse
# import time


class BatterySpider(Spider):
    name = 'battery'
    page_num = 201
    keyword_list = ['电瓶']
    base_url = 'https://search.jd.com/Search?keyword=%s&enc=utf-8&page=%d'

    battery_type = re.compile(r'适用类型：(.*?)')
    battery_series = re.compile(r'产品系列：(.*?)')

    m_type = ''
    m_price = ''
    m_brand = ''
    m_series = ''

    def start_requests(self):
        for keyword in self.keyword_list:
            for page in range(1, self.page_num, 2):
                url = self.base_url % (keyword, page)
                yield Request(url=url, callback=self.parse, meta={'keyword': keyword, 'middleware': True})

    def parse(self, response):
        print('Begin parse', response.url)
        products = response.xpath('.//ul[@class="gl-warp clearfix"]/li')

        for product in products:
            price = ''.join(product.xpath('.//div[@class="p-price"]//i/text()').extract()).strip()
            sku = ''.join(product.xpath('./@data-sku').extract()).strip()
            url = "https://item.jd.com/" + sku + ".html"
            yield Request(url=url, callback=self.parse_subpage, meta={'price': price, 'middleware': False})

    def parse_subpage(self, response):
        self.m_price = response.meta['price']
        self.m_brand = response.xpath('.//ul[@id="parameter-brand"]/li/@title').extract_first()
        parameter_list = response.xpath('.//ul[@class="parameter2 p-parameter-list"]/li/text()').extract()

        for parameter in parameter_list:
            parameter_str = ''.join(parameter).strip()

            m = self.battery_type.search(parameter_str)
            if m:
                print("type:" + parameter_str)
                self.m_type = parameter_str
            n = self.battery_series.search(parameter_str)
            if n:
                print("series:" + parameter_str)
                self.m_series = parameter_str

        item = BatteryItem()
        item['price'] = self.m_price
        item['brand'] = self.m_brand
        item['type'] = self.m_type
        item['series'] = self.m_series
        yield item
        print(item)
